package cn.stacey.day12;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.junit.Test;

public class DemoCrawler {
	private static final String URL_PATTERN = "((ht|f)tps?):\\/\\/[\\w\\-]+(\\.[\\w\\-]+)+([\\w\\-"
			+ "\\.,@?^=%&:\\/~\\+#]*[\\w\\-\\@?^=%&\\/~\\+#])?";
//	private static final String URL_PATTERN = "((ht|f)tps?):\\/\\/[\\w\\-]+(\\.[\\w\\-]+)+([\\w\\-"
//			+ "\\.,@?^=%&:\\/~\\+#]*[\\w\\-\\@?^=%&\\/~\\+#])?";
	@Test
	public void test() {
		parseline("http://www.baidu.com add");
	}
	public static void main(String[] args) {
		try {
			URL url=new URL("http://www.jikedaohang.com");
			InputStream webStream=url.openStream();
			BufferedReader reader=new BufferedReader(new InputStreamReader(webStream,"utf-8"));
			String line=null;
			while((line=reader.readLine())!=null) {
//				System.out.println(line);
				parseline(line);
			}

		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
	public static void parseline(String line) {
		//使用正则表达式去字符串匹配里面的子串
		Pattern pattern=Pattern.compile(URL_PATTERN);
		//每次匹配到的字符串
		Matcher matcher=pattern.matcher(line);
		while(matcher.find()) {
			//找到匹配字符串，取出匹配字符串
			System.out.println(matcher.group());
		}
;	}
}
